#!/usr/bin/env python
# -*-coding:UTF-8 -*-
'''
@Project ：19-xpath站长素材.py
@File：42-异步爬虫百度小说.py
@Author ：文非
@Date：2021/3/1520:52
@Require：# 百度小说西游记的爬取
'''

# http://dushu.baidu.com/api/pc/getCatalog?data={“book_id”:“4306063500”} 第一个url "代表引号 --- 所有章节的名称
# http://dushu.baidu.com/api/pc/getChapterContent?data={"book_id":"4306063500","cid":"4306063500|11348657","need_bookinfo":1}-#小说内容
import requests
import asyncio
import aiohttp
import json
import aiofiles
import time
# UA伪装
headers = {
        'User-Agent': "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"
    }

async def get_content(cid,book_id,title):
    """异步"""
    data = {
        "book_id": book_id,
        "cid": f"{book_id}|{cid}",
        "need_bookinfo": 1
    }
    # 将json数据类型转换为字符串
    data = json.dumps(data)
    url = f"http://dushu.baidu.com/api/pc/getChapterContent?data={data}"
    async with aiohttp.ClientSession() as session:
        async with session.get(url)as result:
            novel = await result.json()
            novel_content = novel["data"]["novel"]["content"]
            path="./西游记/"+title
            async with aiofiles.open(path,mode="w",encoding="utf-8")as fp:
                await fp.write(novel_content)

async def get_cid():
    """同步"""
    # 指定url
    params = {
        "data": '{"book_id":"4306063500"}'
    }
    url = 'http://dushu.baidu.com/api/pc/getCatalog?'
    # 发送请求获取内容
    data_dic = requests.get(url=url, params=params, headers=headers).json()
    # print(data_dic) # str类型
    print(type(data_dic))

    tasks=[] # 准备空列表准备进行异步任务
    book_id = "4306063500"
    for item in data_dic['data']['novel']['items']:  #  对应每一个章节的名称和id
        title = item["title"]
        cid = item["cid"]
        # 每个url相当于一个异步任务 准备异步操作
        # tasks.append({"cid":cid, "book_id":book_id,"title":title})
        tasks.append(get_content(cid, book_id, title))

    await asyncio.wait(tasks)

# def get_cid():
#     """同步"""
#     # 指定url
#     params = {
#         "data": '{"book_id":"4306063500"}'
#     }
#     url = 'http://dushu.baidu.com/api/pc/getCatalog?'
#     # 发送请求获取内容
#     data_dic = requests.get(url=url, params=params, headers=headers).json()
#     # print(data_dic) # str类型
#     print(type(data_dic))
#
#     tasks = []  # 准备空列表准备进行异步任务
#     book_id = "4306063500"
#     for item in data_dic['data']['novel']['items']:  # 对应每一个章节的名称和id
#         title = item["title"]
#         cid = item["cid"]
#         # 每个url相当于一个异步任务 准备异步操作
#         # tasks.append({"cid":cid, "book_id":book_id,"title":title})
#         tasks.append(get_content(cid, book_id, title))
#     asyncio.run(asyncio.wait(tasks))

def main():
    # 1 同步操作，访问get_cid 拿到所有章节的cid和名称
    print(time.time())
    asyncio.run(get_cid())
    print(time.time())
    # 2 异步操作，访问get_content 下载所有文章内容
if __name__=="__main__":
    main()